
(defun List-variables-correlated (variables)
"Args:VARIABLES: A list of names of variables. Returns a list with the combinations of variables, and two lists with the index of variables in the original list to facilitate the location of original variables."
  (let* ((varcorrelated nil)
         (xcorrelated nil)
         (ycorrelated nil)
         (i nil)
         (j nil)
         (variables variables)
         (nvariables (length variables))
         )

    (dotimes (i nvariables)
             (dotimes
              (j i)
              (setf varcorrelated 
                    (append varcorrelated 
                            (list 
                             (concatenate 'string 
                                          (select variables i)"-" (select variables j)))))
              (setf xcorrelated (append xcorrelated (list  j)))
              (setf ycorrelated (append ycorrelated (list  i))) ;esto genera las dos listas de indices para luego recuperar mas facilmente las variables originales
              ))
    (list varcorrelated xcorrelated ycorrelated))
  )
(defun non-missing (var) 
  "Args: VAR. Takes a var with missing values and gives the var without them "
  (select var (which (map-elements 'not (map-elements 'equal nil var)))))

(defun id-missing (var) 
  "Args: VAR. Takes a var with missing values and gives the position of them"
  (which  (map-elements 'equal nil var)))

(defun id-non-missing (var)
  "Args: VAR. Takes a var with missing values and gives the position of the non-missing"
  (which (map-elements 'not (map-elements 'equal nil var))))

(defun list-missing (data)
    "Args: DATA. Returns a list of lists of positions of missing data in DATA"
  
  (map-elements #'id-missing (column-list data))
       
)

(defun matrix-to-list (correlations-matrix)
  (let*
    (
     (correlations-matrix correlations-matrix)
     (nvariables (array-dimension correlations-matrix 0))
     (listcorrelations nil)    
     )
    
    (dotimes (h nvariables)
             (dotimes (k nvariables)
                      (if (> h k)
                          (setf listcorrelations 
                                (append listcorrelations                                                 
                                        (list 
                                         (select correlations-matrix h k))))
                          )
                              
                      ))
    listcorrelations))


(defmeth mv-data-object-proto :report-missing-data ()
  (let* ((data-type (send self :data-type))
         (result-list)
         (loc)
         (miss-values (missingp (send self :active-data-matrix '(numeric))))
         (result 
          (if (not miss-values)
              (send self :describe-data 
                    (column-list (send self :active-data-matrix '(all)))
                    (send self :active-variables '(all)) 
                    (list (list 0 1 2)))
              (combine 
               (choose-subset-dialog 
                "Choose summary information"
                (list
                 "Descriptives"
                 "Correlations"
                 "Comparison of descriptives by missing-observed")
                )))
          ))
    (when miss-values     
          (when result          
                (when (member '0 result)
                      (send self :descriptive-missing))
                (when (member '1 result)
                      (send self :corr-missing))
                (when (member '2 result)
                      (send self :descriptives-by-missing))))
            
    ))





 (defun report-missing ()
   (send current-data :descriptive-missing))
           
               


(defmeth mv-data-object-proto :descriptive-missing ()
   "Computes descriptive statistics. This function is designed by internal use and called by the analysis method of the missing-data-object-proto"
    (let* (
           (data (send current-data :active-data-matrix '(numeric)))
           (variables (send current-data :active-variables '(numeric)))
           (n (array-dimension data 0))
          ; (missing-by-var (mapcar #'(lambda (var) (length (combine (list-missing data)))) (column-list data)))
           (cases-missing (length (remove 'nil (combine (list-missing data)))))
          (variables variables)
          (uniwisedata nil)
          (listwisedata nil)
          (mediauniw nil)
          (medianauniw nil)
          (nuniw nil)
          (medialist nil)
          (medianalist nil)
          (nlist nil)
          (listwisedatamatrix (remove-missing-data-rows data))
          (casos nil)
          (stdvlistwise nil)
          (stduniwise nil)
          (minlistwise nil)
          (minuniwise nil)
          (maxuniwise nil)
          (maxlistwise nil)
          (skewuniwise nil)
          (skewlistwise nil)
          (kurtosisuniwise nil)
          (kurtosislistwise nil)
          (casos n)
          (data data)
        ;  (cases-missing (length (combine (non-missing missing-by-var))))
          (matrix-size (* n (length variables)))

          )
    
      (setf w (report-header 
                               (strcat (send self :title) " - Descriptives for missing data")
                               :page t))
      (send w :size 600 200)
      (send self :data-info w)
      (display-string 
      	(format nil "~%~20a  " "DESCRIPTIVES FOR MISSING DATA") w)
      (display-string (format nil "~%~20a  " "") w)
      (display-string  
       (format nil  "~%~35a ~9,2f" "Number of cases" casos) w)
      (display-string    
       (format nil  "~%~35a ~9,2f" "Number of values in the data matrix" matrix-size) w)
      (display-string    
       (format nil "~%~35a ~9,2f" "Number of values missing" cases-missing) w)
      (display-string    
       (format nil "~%~35a ~9,2f" "Percentage of values missing" 
               (* 100 (/ cases-missing matrix-size))) w)
      (display-string    
       (format nil  "~%~16a ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~9,2f" "Descriptives" "Mean" "Median" "Std" "Skew" "Kurt" "Min" "Max" "Missing" "N") w )

   (dotimes (i (array-dimension data 1))

            ;Aqui lo relacionado con uniwise
            (setf uniwisedata (non-missing 
                         (col 
                          data
                          i)))
            (setf mediauniw (mean uniwisedata));media normal
            (setf medianuniw (median uniwisedata))
            (setf nuniw (length uniwisedata))
            (setf stduniwise (standard-deviation uniwisedata))
            (setf minuniwise (min uniwisedata))
            (setf maxuniwise (max uniwisedata))
            (setf skewuniwise (skewness uniwisedata))
            (setf kurtosisuniwise (kurtosis uniwisedata))

            ;Comienza lo relacionado con listwise
            (setf listwisedata (if listwisedatamatrix (col listwisedatamatrix i) nil))
            (setf medialist (if listwisedatamatrix (mean listwisedata) nil))
            (setf medianalist (if listwisedatamatrix (median listwisedata) nil))
            (setf nlist (if listwisedata (length listwisedata) nil))
            (setf stdlistwise (if listwisedata (standard-deviation listwisedata) nil))
            (setf minlistwise  (if listwisedata (min listwisedata) nil))
            (setf maxlistwise (if listwisedata (max listwisedata) nil))
            (setf skewlistwise (if listwisedata (skewness listwisedata) nil))
            (setf kurtosislistwise (if listwisedata (kurtosis listwisedata) nil))
            ;aqui empieza lo de la impresion

          (display-string  
                      (format nil "~%~20a" (select variables i)) w)
            (display-string   
             (format nil "~%~0a ~12a ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f"
                     "" "Uniwise" mediauniw medianuniw  
                     stduniwise skewuniwise kurtosisuniwise minuniwise 
                     maxuniwise (- casos nuniw) nuniw)w )          
            (display-string  
             (format nil "~%~0a ~12a ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f" 
                             "" "Listwise" medialist medianalist stdlistwise 
                             skewlistwise kurtosislistwise minlistwise  
                             maxlistwise (if nlist (- casos nlist) nil) nlist) w)




)
(send w :fit-window-to-text)
))



 (defun corr-missing ()
   (send current-data :corr-missing))
           
               
(defun listwisecorrelations2 (listwisedata)
  "Args. LISTWISEDATA. Data without missing. Prints correlation matrix as a list"
  (matrix-to-list (correlation-matrix listwisedata))
  )

(defmeth mv-data-object-proto :corr-missing () 
  
   "Prints correlations. This function is designed by internal use and called by the analysis method of the missing-data-object-proto"
  (let* (
         (data (send current-data :active-data-matrix '(numeric)))
         (variables (send current-data :active-variables '(numeric)))
         (infopatterns (cases-in-missing-patterns2 data))
         (corrlistwise nil)         
         (listwise-data (remove-missing-data-rows data))
         (listwise-data (if listwise-data 
                            (if (= (array-dimension listwise-data 0) 0) nil) 
                            listwise-data))
         (infopairwise (pairwisecorrelations data (list-missing data)))
         (corrpairwise (matrix-to-list (first infopairwise)))
         (NPairwise (second infopairwise))
         (N1 (third infopairwise))
         (n2 (fourth infopairwise))
         (NListwise (if listwise-data (array-dimension listwise-data 0) nil))
         (corrlistwise (if listwise-data (listwisecorrelations2 listwise-data) nil))
         (ntotal (array-dimension data 0))
        ; (correlations-missing (corrmissing2 (fourth infopatterns)))
         (correlations-missing (matrix-to-list (listwisecorrelations (fourth infopatterns))))
         (listvariables (first (list-variables-correlated variables))) ;no queria hacerlo asi 
         ;pero parece ser la unica manera de hacerlo funcionar
         

         )
    (setf w (report-header 
                               (strcat (send self :title) " - Correlations for missing data")
                               :page t))
    
    (display-string
    	(format nil "~%~20a  " "PAIRWISE AND LISTWISE CORRELATIONS") w)
    (display-string (format nil "~%~20a  " "") w)
    (display-string 
     (format nil "~%~20a ~5,2f" "Ntotal" ntotal) w)
	
    (display-string 
     (format nil "~%~20a ~5,2f" "NListwise" NListwise) w)
    (display-string  
     (format nil "~%~32a ~9a ~8a ~8a ~7a  ~6a ~6a" "Variables"  "Pair" 
             "List" "N PW" "N V1" "N V2" "CorrMiss" ) 
      w)
    (dotimes (i (length listvariables))
         (display-string 
          (format nil "~%~28a ~8,2f  ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f" 
                  (select listvariables i) 
                  (select corrpairwise i) 
                  (if listwise-data (select corrlistwise i) nil)                       
                  (select Npairwise i) 
                  (select n1 i)
                  (select n2 i)
                  (select correlations-missing i)
                  ) w)
             )
(send w :fit-window-to-text)
    ))

(defmeth mv-data-object-proto :descriptives-by-missing ()
  (let* (
        (data (column-list (send self :active-data-matrix '(numeric))))
        (missing-by-var 
         (list-missing (send self :active-data-matrix '(numeric))))
        (ncases (length (first data)))
         (ncol (length data))
         (present-part nil)
         (missing-part nil)
         (missbyvar nil)
         (presbyvar nil)
         (variables (send self :active-variables '(numeric)))
        )
    (setf w (report-header 
                   (strcat (send self :title) " - DESCRIPTIVES BY VARIABLE WITH MISSING VALUES")
             :page t))
    (display-string 
                   (format nil "~%~20a  " "DESCRIPTIVES COMPARING VARIABLES FOR MISSING AND OBSERVED PARTS OF VARIABLES") w)
    (display-string    
     (format nil  "~%~16a ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~9,2f" "Descriptives" "Mean" "Median" "Std" "Skew" "Kurt" "Min" "Max" "Missing" "N") w )

    (mapcar #'(lambda (elem)

            (when (select missing-by-var elem)
                  (setf missbyvar (select missing-by-var elem))
                 (setf presbyvar (set-difference (iseq ncases) missbyvar ))
                  (terpri)
                  (display-string
                   (format nil "~%~%~20a~%  " (strcat "   " (select variables elem) ) ) w)
    (display-string    
     (format nil  "~%~16a ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~9,2f" "Descriptives" "Mean" "Median" "Std" "Skew" "Kurt" "Min" "Max" "Missing" "N") w )
                  
                  (mapcar #'(lambda (vars elem2)
                              (when (not (equal missbyvar
                                                (select missing-by-var elem2)))
                                    
                                    (display-string    
                                     (format nil  "~%~16a " (select variables elem2)) w )
                                    (setf missing-part (remove 'nil (select vars missbyvar)))
                                    (when (> (length missing-part) 0)
                                          (display-string  
                                           (format nil "~%~0a ~12a ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f" 
                                                   "" "MISSING" 
                                                   (mean missing-part)
                                                   (median missing-part) 
                                                   (if (=
                                                   (length  (remove-duplicates missing-part)) 1)
                                                   "     -"
                                                   (standard-deviation missing-part))
                                                   (if (<
                                                        (length  (remove-duplicates missing-part)) 3)
                                                     "     -"
                                                      (skewness missing-part))
                                                   (if 
                                                    (<
                                                     (length  (remove-duplicates missing-part)) 3)
                                                      "     -"
                                                    (kurtosis missing-part))
                                                   (min missing-part)  
                                                   (max missing-part) 
                                                   (- (length missbyvar) (length missing-part))
                                                   (length missing-part)) w))


                                    (setf present-part 
                                          (remove 'nil 
                                                  (select vars
                                                          presbyvar)))
                                     (when (> (length present-part) 0)
                                                        (display-string  
                                           (format nil "~%~0a ~12a ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f ~8,2f" 
                                                   "" "OBSERVED" 
                                                   (mean present-part)
                                                   (median present-part) 
                                                   (if (=
                                                   (length  (remove-duplicates present-part)) 1)
                                                   "     -"
                                                   (standard-deviation present-part))
                                                   (if (= 
                                                        (length  (remove-duplicates present-part)) 1)
                                                      "     -"
                                                      (skewness present-part))
                                                   (if (= 
                                                        (length  (remove-duplicates present-part)) 1)
                                                       "     -"
                                                       (kurtosis present-part))
                                                   (min present-part)  
                                                   (max present-part) 
                                                   (- (length presbyvar) (length present-part))
                                                   (length present-part)) w))
                                    ))
                             (select data (remove elem (iseq (length data))))
                            (remove elem (iseq ncol)))
                ))
            (iseq ncol))
    (send w :fit-window-to-text)
    nil
    ))



(defun corrmissing2 (missingness-matrix)
  "ARGS: A matrix of ones and zeros. Computes correlations and returns the lowest triangle as a list. Controls for variables with no variance and prints NIL."
  (let* (
        (missingness-matrix missingness-matrix)
        (casos (array-dimension missingness-matrix 0))
         (numvariables (array-dimension missingness-matrix 1))
         (corrmissinglist nil)
        )
    (dotimes (i numvariables)
             (dotimes (j numvariables)
                      
                    (if (> i j)
                        
                          (if (and 
                               
                               (< (sum (col missingness-matrix i)) casos)
                               (< (sum (col missingness-matrix j)) casos)
                               (> (sum (col missingness-matrix i)) 0)
                               (> (sum (col missingness-matrix j)) 0)
                               )
                              (setf corrmissinglist  
                                    (append corrmissinglist 
                                            (list 
                                             (listwisecorrelations 
                                              (bind-columns 
                                               (col missingness-matrix i)
                                               (col missingness-matrix j)
                                               ))))) 
                              (setf corrmissinglist 
                                    (append corrmissinglist 
                                            (list nil)))
                              ))
                      ))
 corrmissinglist))



(defun pairwisecorrelations2 (data missing-by-var)
  "Args. DATA. Data with missing data. Missing-by-var. A list of lists of missing data in each variable. Returns
a list with four lists: pairwise correlations, n pairwise y n by variable"
  
  (let* (
         (data data)
         (missing-by-var missing-by-var)
         (i nil)
         (j nil)
         (h nil)
     
         (pairw nil)
         (n1 nil)
         (n2 nil)
         (secuencia nil)
         (paired-list nil)
         (cases (select (array-dimensions data ) 0))
         (nvars (select (array-dimensions data ) 1))
         (index-list (iseq 0 (-  cases 1)))
         (corrpairwise-matrix (make-array (list nvars nvars)))
         (npairwise-matrix (make-array (list nvars nvars)))
         (n1-matrix (make-array (list nvars nvars)))
         (n2-matrix (make-array (list nvars nvars)))
         (temp nil)
         )
    
    (dotimes (i nvars)
             (dotimes (j i)
                      (setf paired-list  
                                         (union (select missing-by-var i)
                                                 (select missing-by-var j)))
                      (setf secuencia (set-difference index-list
                                        paired-list))
                     
                     (setf temp (covariance-matrix (select data secuencia (list i j))))

                     (setf (select corrpairwise-matrix i j)
                           (/ (select temp 0 1) (* (sqrt (select temp 0 0)) (sqrt (select temp 1 1)))))
                      
                    
                      (setf (select npairwise-matrix i j) (length secuencia))
                      (setf (select n1-matrix i j)  (- cases (length 
                                               (select missing-by-var i))))
                      (setf (select n2-matrix i j)  (- cases (length 
                                               (select missing-by-var j))))

                      
                      
                      ))
    
    (list (select (combine corrpairwise-matrix) (which (combine corrpairwise-matrix))) 
          (select (combine npairwise-matrix) (which (combine npairwise-matrix))) 
          (select (combine n1-matrix) (which (combine n1-matrix)))
          (select (combine n2-matrix) (which (combine n2-matrix)))
          )
    )
  )

